import numpy as np
import matplotlib.pyplot as plt
from sem_dt_rf.decision_tree.decision_tree import RegressionDecisionTree
from sklearn.tree import DecisionTreeRegressor
import seaborn as sns
%matplotlib inline

# generate example
x_shape = 300
# features for fit 
x = np.arange(x_shape) / 100
# features for test the fitted model
x_test = np.random.uniform(0, 3.0, 300)#np.arange(x_shape) / 100#np.random.uniform(0, 3, 300)
# values for fit
y = x**3 * np.sin(x**3) + (np.random.random(x_shape)*2 - 1)  * 3 
#values with the correct dependence on x (without error)
y_true_func = x_test**3 * np.sin(x_test**3)

for i in [4,6,11,12, 30]:
    max_depth = i
    reg_tree = RegressionDecisionTree(max_depth=max_depth, min_leaf_size = 2) 
    reg_tree.fit(x.reshape((-1,1)), y)
    
    skl = DecisionTreeRegressor(max_depth = max_depth)
    skl.fit(x.reshape((-1,1)), y)
    y_pred = reg_tree.predict(x.reshape((-1,1)))
    y_test = reg_tree.predict(x_test.reshape((-1,1)))
    skl_pred = skl.predict(x.reshape((-1,1)))
    skl_test = skl.predict(x_test.reshape((-1,1)))

    fig, ax = plt.subplot_mosaic([['train', 'test'],['sklearn train', 'sklearn test']])
    fig.set_size_inches(18, 12)
    fig.suptitle(f'Max depth ={i} ', fontsize=18)
    
    ax['train'].scatter(x, y, label='y generated')
    ax['train'].scatter(x,y_pred, label = 'y predicted')
    ax['train'].legend(fontsize=14)
    ax['train'].set_title('The result of a tree on the fitting data', fontsize=18)

    ax['test'].scatter(x_test, y_true_func, label='y = x^3 + sin(x^3)')
    ax['test'].scatter(x_test, y_test, label='y predicted')
    ax['test'].legend(fontsize=14)
    ax['test'].set_title('The result of a tree on new generated data', fontsize=18)

    ax['sklearn train'].scatter(x, y, label='y generated')
    ax['sklearn train'].scatter(x,skl_pred, label = 'y predicted')
    ax['sklearn train'].legend(fontsize=14)
    ax['sklearn train'].set_title('The result of a sklearn tree on the fitting data', fontsize=18)

    ax['sklearn test'].scatter(x_test, y_true_func, label='y = x^3 + sin(x^3)')
    ax['sklearn test'].scatter(x_test, skl_test, label = 'sklearn predicted')
    ax['sklearn test'].legend(fontsize=14)
    ax['sklearn test'].set_title('The result of a sklearn tree on new generated data', fontsize=18)


    
    
    plt.show()

import sys
#sys.path.extend(['']) # change your path

from sem_dt_rf.random_forest.random_forest import RandomForestClassifier as MyRandomForestClassifier

from sklearn.datasets import fetch_covtype
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import balanced_accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.multiclass import OneVsRestClassifier
import lightgbm as lgb
import xgboost as xgb
import pandas as pd

cov_type = fetch_covtype(as_frame = True)

print(cov_type.DESCR)
cov_type_train_columns = cov_type.frame.columns.to_list()
cov_type_train_columns.remove('Cover_Type')

.. _covtype_dataset:

Forest covertypes
-----------------

The samples in this dataset correspond to 30Г—30m patches of forest in the US,
collected for the task of predicting each patch's cover type,
i.e. the dominant species of tree.
There are seven covertypes, making this a multiclass classification problem.
Each sample has 54 features, described on the
`dataset's homepage <https://archive.ics.uci.edu/ml/datasets/Covertype>`__.
Some of the features are boolean indicators,
while others are discrete or continuous measurements.

**Data Set Characteristics:**

    =================   ============
    Classes                        7
    Samples total             581012
    Dimensionality                54
    Features                     int
    =================   ============

:func:`sklearn.datasets.fetch_covtype` will load the covertype dataset;
it returns a dictionary-like 'Bunch' object
with the feature matrix in the ``data`` member
and the target values in ``target``. If optional argument 'as_frame' is
set to 'True', it will return ``data`` and ``target`` as pandas
data frame, and there will be an additional member ``frame`` as well.
The dataset will be downloaded from the web if necessary.

x_train, x_valtest, y_train, y_valtest = train_test_split(cov_type.frame[cov_type_train_columns],
                                                    cov_type.frame[['Cover_Type']], 
                                                    test_size = 0.33
                                                   )
x_val, x_test, y_val, y_test = train_test_split(x_valtest,
                                                    y_valtest, 
                                                    test_size = 0.5
                                                   )

import optuna
from functools import partial
import numpy as np


def optimize(objective, X_train, y_train, X_val, y_val, 
             n_trials, n_jobs, direction,
             n_startup_trials, gamma, n_ei_candidates, multivariate
            ):
    tpe_sampler = optuna.samplers.TPESampler(
        n_startup_trials=n_startup_trials, 
        gamma=gamma,
        n_ei_candidates=n_ei_candidates,
        multivariate=multivariate
    )
    study = optuna.create_study(sampler=tpe_sampler, direction=direction)
    if X_val is None or y_val is None:
        obj_func = partial(objective, X_train=X_train, y_train=y_train)
    else:
        obj_func = partial(objective, X_train=X_train, X_val=X_val, y_train=y_train, y_val=y_val)
    study.optimize(obj_func, n_trials=n_trials, n_jobs=n_jobs)
    return study

# sklearn RF
def objective_RFC(
    trial: optuna.trial.Trial, # нечто оптюновское
    X_train, X_val, y_train, y_val # данные
):

    model = OneVsRestClassifier(RandomForestClassifier(
        n_estimators = 20,#trial.suggest_int('n_estimators', 10, 150, log=True),
        max_depth = None, #trial.suggest_int('max_depth', 10, 50),
        #min_samples_split = 1, #trial.suggest_int('min_samples_split', 5, 100),
        min_samples_leaf = 1, # trial.suggest_int('min_samples_leaf', 5, 100),
        max_features =  trial.suggest_int('max_features', 5, 54) ,
        #max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 5, 300, log = True),
        #min_impurity_decrease = trial.suggest_float('min_impurity_decrease', 0, 0.1),
        #bootstrap = True,
        #max_samples =  trial.suggest_float('max_samples', 0.5, 1, step = 0.1)
        
    ))
    
    model.fit(X_train, y_train)
    y_predict = model.predict_proba(X_val).argmax(axis=1) + 1
    score = balanced_accuracy_score(y_val, y_predict)
    return score


params_RFC = dict(
    objective = objective_RFC,
    X_train = x_train,
    y_train = y_train,
    X_val = x_val,
    y_val = y_val,
    n_trials = 49,
    n_jobs = 15,
    direction = 'maximize',
    n_startup_trials = 49 ,
    gamma = lambda n_trials: min(int(np.ceil(0.1 * n_trials)), 25),
    n_ei_candidates = 5,
    multivariate = True
)

study_RFC = optimize(**params_RFC)

# How to look at study
from optuna.visualization import plot_slice
# the func below is needed to choose the parts to see
def my_clipper(trial: optuna.trial.FrozenTrial):
    score = trial.value
    return min(1.1, score)
plot_slice(study_RFC) #, target= my_clipper)
# Нужно max features и все, остальное дает заметно меньшие справления
# max_features = 26, не лучший, но бьет 0.92 и скорость важна
# Лучшее 45

model_myRFC =  MyRandomForestClassifier(
        n_estimators=20,
        max_objects_samples=1,
        max_features_samples=45/54,
        #max_depth= 40,
        #min_samples_leaf=10,  
    )

model_myRFC.fit(x_train.to_numpy(), y_train.to_numpy())
y_predict_myRFC = model_myRFC.predict(x_val.to_numpy())
score = balanced_accuracy_score(y_val-1, y_predict_myRFC)
print('No OneVsRestClassifier', score)

0.7437730120814622

model_myRFC =  OneVsRestClassifier(MyRandomForestClassifier(
        n_estimators=20,
        max_objects_samples=1,
        max_features_samples=45/54,
        #max_depth= 40,
        #min_samples_leaf=10,  
    ))

model_myRFC.fit(x_train.to_numpy(), y_train.to_numpy())
y_predict_myRFC = model_myRFC.predict(x_val.to_numpy())
score = balanced_accuracy_score(y_val, y_predict_myRFC)
print('With OneVsRestClassifier', score)

0.7965172278958403

# lightgbm
def objective_lightgbm(
    trial: optuna.trial.Trial, # нечто оптюновское
    X_train, X_val, y_train, y_val # данные
):

    model = lgb.LGBMModel(
        boosting_type = 'rf',
        objective = 'multiclass',
        subsample_freq = 1,
        subsample = 0.99999,
        colsample_bytree = trial.suggest_float('max_features', 4/54, 1, step = 0.03 ),
        max_depth = -1,
        num_class = 7,
        n_estimators = 20,
        num_leaves = 2**10,
        learning_rate = 1,
        min_child_samples = 1,
    )
    
    model.fit(X_train, y_train)
    y_predict = model.predict(X_val).argmax(axis=1)#model.predict_proba(X_val).argmax(axis=1) + 1
    score = balanced_accuracy_score(y_val, y_predict)
    return score


params_lightgbm = dict(
    objective = objective_lightgbm,
    X_train = x_train,
    y_train = y_train-1,
    X_val = x_val,
    y_val = y_val-1,
    n_trials = 30,
    n_jobs = 15,
    direction = 'maximize',
    n_startup_trials = 30 ,
    gamma = lambda n_trials: min(int(np.ceil(0.1 * n_trials)), 25),
    n_ei_candidates = 5,
    multivariate = True
)

study_lightgbm = optimize(**params_lightgbm)

from optuna.visualization import plot_slice
# the func below is needed to choose the parts to see
def my_clipper(trial: optuna.trial.FrozenTrial):
    score = trial.value
    return min(1.1, score)
plot_slice(study_lightgbm)

# Проверка результата с тестовым значением num_leaves
model_lgbm = lgb.LGBMModel(
        boosting_type = 'rf',
        objective = 'multiclass',
        subsample_freq = 1,
        subsample = 0.99999,
        colsample_bytree = 0.82,
        max_depth = -1,
        num_class = 7,
        n_estimators = 20,
        num_leaves = 2**15,
        learning_rate = 1,
        #min_child_samples = 1,
        #num_parallel_tree = 100,
    )
model_lgbm.fit(x_train, y_train-1)
y_predict_lgbm = model_lgbm.predict(x_val).argmax(axis=1)#model.predict_proba(X_val).argmax(axis=1) + 1
score = balanced_accuracy_score(y_val-1, y_predict_lgbm)
print(score)

[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.001626 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 2263
[LightGBM] [Info] Number of data points in the train set: 389278, number of used features: 53
[LightGBM] [Info] Start training from score -1.007974
[LightGBM] [Info] Start training from score -0.718560
[LightGBM] [Info] Start training from score -2.789077
[LightGBM] [Info] Start training from score -5.349108
[LightGBM] [Info] Start training from score -4.121841
[LightGBM] [Info] Start training from score -3.509073
[LightGBM] [Info] Start training from score -3.346168
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
0.9351298964757427

# xgboost
def objective_xgb(
    trial: optuna.trial.Trial, # нечто оптюновское
    X_train, X_val, y_train, y_val # данные
):

    params = {
      #"colsample_bynode": 45/54,
      "booster" : 'gbtree',
      "learning_rate": 1,
      "max_depth": 0,
      "num_parallel_tree": 20,
      "objective": "multi:softmax",
      "subsample": 0.9999,
      "tree_method": "hist",
      "device": "cpu",
      "verbosity" : 2,
      "n_estimators" : 1,
    }
    
    model = xgb.XGBClassifier(
       **params,
       colsample_bynode = trial.suggest_float('max_features', 4/54, 1, step = 0.03 ),
       min_child_weight = trial.suggest_float('min_child_weight', 0, 2 )
               
    )
    
    model.fit(X_train, y_train)
    y_predict = model.predict(X_val)#.argmax(axis=1)#model.predict_proba(X_val).argmax(axis=1) + 1
    score = balanced_accuracy_score(y_val, y_predict)
    return score


params_xgb = dict(
    objective = objective_xgb,
    X_train = x_train,
    y_train = y_train-1,
    X_val = x_val,
    y_val = y_val-1,
    n_trials = 80,
    n_jobs = 3,
    direction = 'maximize',
    n_startup_trials = 60 ,
    gamma = lambda n_trials: min(int(np.ceil(0.1 * n_trials)), 25),
    n_ei_candidates = 5,
    multivariate = True
)

study_xgb = optimize(**params_xgb)

from optuna.visualization import plot_slice
# the func below is needed to choose the parts to see
def my_clipper(trial: optuna.trial.FrozenTrial):
    score = trial.value
    return min(1.1, score)
plot_slice(study_xgb)

import time


models = {
    "RFC" : OneVsRestClassifier(RandomForestClassifier(
                n_estimators = 20,
                max_depth = None, 
                min_samples_leaf = 1,
                max_features =  45 ,
            )),
    "myRFC" : OneVsRestClassifier(MyRandomForestClassifier(
                n_estimators=20,
                max_objects_samples=1,
                max_features_samples=0.83,
            )),
    "lightgbm" : lgb.LGBMModel(
                boosting_type = 'rf',
                objective = 'multiclass',
                subsample_freq = 1,
                subsample = 0.9999,
                colsample_bytree = 0.82,
                max_depth = -1,
                num_class = 7,
                n_estimators = 20,
                num_leaves = 2**15,
                learning_rate = 1,
                min_child_samples = 1,
            ),
    "xgboost" : xgb.XGBClassifier(
                booster = 'gbtree',
                colsample_bynode = 0.85,
                min_child_weight = 0.2,
                learning_rate = 1,
                max_depth = 0,
                num_parallel_tree = 20,
                objective = "multi:softmax",
                subsample = 0.9999,
                tree_method = "hist",
                device = "cpu",
                verbosity = 2,
                n_estimators = 1,
                n_jobs = 3
            )  
}

#models_analis = {}

x_train_np = x_train.to_numpy()
y_train_np = y_train.to_numpy() - 1 
x_test_np = x_test.to_numpy()
y_test_np = y_test.to_numpy() - 1

for model_name, model in models.items():
    if model_name in ["RFC", "myRFC", "lightgbm"]: # adjust !
        continue
    print(f"Start fitting {model_name}")
    start_fit = time.time()
    model.fit(x_train_np, y_train_np)
    end_fit = time.time()
    print(f"End fitting {model_name}")

    print(f"Start predicting {model_name}")
    start_predict = time.time()
    if model_name == "lightgbm":
        y_predict = model.predict(x_test_np).argmax(axis=1)
    else:
        y_predict = model.predict(x_test_np)
    end_predict = time.time()
    print(f"End predicting {model_name}")
    
    score = balanced_accuracy_score(y_test_np, y_predict)

    models_analis[model_name] = {
        'score' : score,
        'fit_time' : end_fit - start_fit,
        'predict_time' : end_predict - start_predict
    }

Start fitting xgboost
End fitting xgboost
Start predicting xgboost
End predicting xgboost

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

models_frame = pd.DataFrame(models_analis).T
models_frame.index.rename('models', inplace= True )
fig, ax = plt.subplot_mosaic([['score'], ['fit time'], ['predict time']])
fig.set_size_inches(13, 15)
ax['score'].set_xticks(np.arange(0, 20, 1)/20)
ax['score'].grid()
ax['score'].set_title('Score')
sns.barplot(models_frame, y = 'models', x = 'score', ax=ax['score'], width = 0.4 , hue = 'models', fill = True, orient = 'h')
ax['fit time'].set_xticks(np.arange(0, 400, 20))
ax['fit time'].grid()
ax['fit time'].set_title('Fit time in seconds')
sns.barplot(models_frame, y = 'models', x = 'fit_time', ax=ax['fit time'], width = 0.4 , hue = 'models', fill = True, orient = 'h')
ax['predict time'].set_xticks(np.arange(0, 4, 0.2))
ax['predict time'].grid()
ax['predict time'].set_title('Predict time in seconds')
sns.barplot(models_frame, y = 'models', x = 'predict_time', ax=ax['predict time'], width = 0.4 , hue = 'models', fill = True, orient = 'h')
plt.show()

models_frame

# dependence on trees number
# evrywhere take only subsample = 0.5 for speed
n_trees = [3,6,9,12,15,18,20,23,26,29,35]
#RFC

RFC_scores = []
for n_t in n_trees:
    print(f"RFC with {n_t} trees is processed")
    model = OneVsRestClassifier(RandomForestClassifier(
                n_estimators = n_t,
                max_depth = None, 
                min_samples_leaf = 1,
                max_features =  45 ,
                bootstrap = True,
                max_samples = 0.5
            ))
    model.fit(x_train_np, y_train_np)
    y_predict = model.predict(x_test_np)
    score = balanced_accuracy_score(y_test_np, y_predict)
    RFC_scores.append(score)


myRFC_scores = []
for n_t in n_trees:
    print(f"myRFC with {n_t} trees is processed")
    model =  OneVsRestClassifier(MyRandomForestClassifier(
                n_estimators=n_t,
                max_objects_samples=0.5,
                max_features_samples=0.83
            ))
    model.fit(x_train_np, y_train_np)
    y_predict = model.predict(x_test_np)
    score = balanced_accuracy_score(y_test_np, y_predict)
    myRFC_scores.append(score)


lightgbm_scores = []
for n_t in n_trees:
    print(f"lightgbm with {n_t} trees is processed")
    model =  lgb.LGBMModel(
               boosting_type = 'rf',
                objective = 'multiclass',
                subsample_freq = 1,
                subsample = 0.5,
                colsample_bytree = 0.82,
                max_depth = -1,
                num_class = 7,
                n_estimators = n_t,
                num_leaves = 2**15,
                learning_rate = 1,
                min_child_samples = 1,
    )
    model.fit(x_train_np, y_train_np)
    y_predict = model.predict(x_test_np).argmax(axis=1)
    score = balanced_accuracy_score(y_test_np, y_predict)
    lightgbm_scores.append(score)


xgb_scores = []
for n_t in n_trees:
    print(f"xgb with {n_t} trees is processed")
    model =   xgb.XGBClassifier(
                booster = 'gbtree',
                colsample_bynode = 0.85,
                min_child_weight = 0.2,
                learning_rate = 1,
                max_depth = 0,
                num_parallel_tree = n_t,
                objective = "multi:softmax",
                subsample = 0.5,
                tree_method = "hist",
                device = "cpu",
                verbosity = 2,
                n_estimators = 1,
                n_jobs = 3
            )  
    model.fit(x_train_np, y_train_np)
    y_predict = model.predict(x_test_np)
    score = balanced_accuracy_score(y_test_np, y_predict)
    xgb_scores.append(score)

fig, ax = plt.subplot_mosaic([['n_trees']])
ax['n_trees'].grid()
sns.scatterplot(x = n_trees, y = RFC_scores, ax= ax['n_trees'], label = 'RFC')
sns.scatterplot(x = n_trees, y = myRFC_scores, ax= ax['n_trees'], label = 'myRFC')
sns.scatterplot(x = n_trees, y = lightgbm_scores, ax= ax['n_trees'], label = 'lightgbm')
sns.scatterplot(x = n_trees, y = xgb_scores, ax= ax['n_trees'], label = 'xgb')
ax['n_trees'].legend(loc='lower left')
ax['n_trees'].set_xlabel("trees number")
ax['n_trees'].set_ylabel("score")
ax['n_trees'].set_title('Dependence on trees number')


#ax['n_trees'].scatter(n_trees, RFC_scores)
#ax['n_trees'].scatter(n_trees, myRFC_scores)
#ax['n_trees'].scatter(n_trees, lightgbm_scores)
#ax['n_trees'].scatter(n_trees, xgb_scores)
plt.show()

	score	fit_time	predict_time
models
RFC	0.925829	242.541658	1.297809
myRFC	0.887439	360.507245	3.902016
lightgbm	0.954323	130.361382	0.277964
xgboost	0.905462	29.415183	0.486007

Домашнее задание (10 баллов)¶

Решение¶

Регрессионое дерево¶

Выводы¶

ВАЖНО¶

Сравнение реализаций RandomForest¶

План¶

Подбор max_features для Random Forest¶

Multiclass для myRFC¶

Подбор max_features для lightgbm¶

Подбор max_features для xgboost¶

max_features около 0.8 для всех моеделей¶

Ниже смотрим на результат всех моделей и визуализируем¶

Выводы¶

Зависимость моделей от количества деревьев¶

Вывод¶